{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "from utils import NerDataset\n",
    "\n",
    "dev_dataset = NerDataset('./processed/processed_dev_bio.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "909"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dev_dataset.sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6306"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset = NerDataset('./processed/processed_training_bio.txt')\n",
    "len(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('[CLS] ， 患 者 3 月 前 因 “ 直 肠 癌 ” 于 在 我 院 于 全 麻 上 行 直 肠 癌 根 治 术 （ D I X O N 术 ） ， 手 术 过 程 顺 利 ， 术 后 给 予 抗 感 染 及 营 养 支 持 治 疗 ， 患 者 恢 复 好 ， 切 口 愈 合 良 好 。 ， 术 后 病 理 示 ： 直 肠 腺 癌 （ 中 低 度 分 化 ） ， 浸 润 溃 疡 型 ， 面 积 3 . 5 * 2 C M ， 侵 达 外 膜 。 双 端 切 线 另 送 “ 近 端 ” 、 “ 远 端 ” 及 环 周 底 部 切 除 面 未 查 见 癌 。 肠 壁 一 站 （ 1 0 个 ） 、 中 间 组 （ 8 个 ） 淋 巴 结 未 查 见 癌 。 ， 免 疫 组 化 染 色 示 ： E R C C 1 弥 漫 （ + ） 、 T S 少 部 分 弱 （ + ） 、 S Y N （ - ） 、 C G A （ - ） 。 术 后 查 无 化 疗 禁 忌 后 给 予 3 周 期 化 疗 ， ， 方 案 为 ： 奥 沙 利 铂 1 5 0 M G O D 1 ， 亚 叶 酸 钙 0 . 3 G + 替 加 氟',\n",
       " [101,\n",
       "  8024,\n",
       "  2642,\n",
       "  5442,\n",
       "  124,\n",
       "  3299,\n",
       "  1184,\n",
       "  1728,\n",
       "  100,\n",
       "  4684,\n",
       "  5499,\n",
       "  4617,\n",
       "  100,\n",
       "  754,\n",
       "  1762,\n",
       "  2769,\n",
       "  7368,\n",
       "  754,\n",
       "  1059,\n",
       "  7937,\n",
       "  677,\n",
       "  6121,\n",
       "  4684,\n",
       "  5499,\n",
       "  4617,\n",
       "  3418,\n",
       "  3780,\n",
       "  3318,\n",
       "  8020,\n",
       "  146,\n",
       "  151,\n",
       "  166,\n",
       "  157,\n",
       "  156,\n",
       "  3318,\n",
       "  8021,\n",
       "  8024,\n",
       "  2797,\n",
       "  3318,\n",
       "  6814,\n",
       "  4923,\n",
       "  7556,\n",
       "  1164,\n",
       "  8024,\n",
       "  3318,\n",
       "  1400,\n",
       "  5314,\n",
       "  750,\n",
       "  2834,\n",
       "  2697,\n",
       "  3381,\n",
       "  1350,\n",
       "  5852,\n",
       "  1075,\n",
       "  3118,\n",
       "  2898,\n",
       "  3780,\n",
       "  4545,\n",
       "  8024,\n",
       "  2642,\n",
       "  5442,\n",
       "  2612,\n",
       "  1908,\n",
       "  1962,\n",
       "  8024,\n",
       "  1147,\n",
       "  1366,\n",
       "  2689,\n",
       "  1394,\n",
       "  5679,\n",
       "  1962,\n",
       "  511,\n",
       "  8024,\n",
       "  3318,\n",
       "  1400,\n",
       "  4567,\n",
       "  4415,\n",
       "  4850,\n",
       "  8038,\n",
       "  4684,\n",
       "  5499,\n",
       "  5593,\n",
       "  4617,\n",
       "  8020,\n",
       "  704,\n",
       "  856,\n",
       "  2428,\n",
       "  1146,\n",
       "  1265,\n",
       "  8021,\n",
       "  8024,\n",
       "  3863,\n",
       "  3883,\n",
       "  3971,\n",
       "  4550,\n",
       "  1798,\n",
       "  8024,\n",
       "  7481,\n",
       "  4916,\n",
       "  124,\n",
       "  119,\n",
       "  126,\n",
       "  115,\n",
       "  123,\n",
       "  145,\n",
       "  155,\n",
       "  8024,\n",
       "  909,\n",
       "  6809,\n",
       "  1912,\n",
       "  5606,\n",
       "  511,\n",
       "  1352,\n",
       "  4999,\n",
       "  1147,\n",
       "  5296,\n",
       "  1369,\n",
       "  6843,\n",
       "  100,\n",
       "  6818,\n",
       "  4999,\n",
       "  100,\n",
       "  510,\n",
       "  100,\n",
       "  6823,\n",
       "  4999,\n",
       "  100,\n",
       "  1350,\n",
       "  4384,\n",
       "  1453,\n",
       "  2419,\n",
       "  6956,\n",
       "  1147,\n",
       "  7370,\n",
       "  7481,\n",
       "  3313,\n",
       "  3389,\n",
       "  6224,\n",
       "  4617,\n",
       "  511,\n",
       "  5499,\n",
       "  1880,\n",
       "  671,\n",
       "  4991,\n",
       "  8020,\n",
       "  122,\n",
       "  121,\n",
       "  702,\n",
       "  8021,\n",
       "  510,\n",
       "  704,\n",
       "  7313,\n",
       "  5299,\n",
       "  8020,\n",
       "  129,\n",
       "  702,\n",
       "  8021,\n",
       "  3900,\n",
       "  2349,\n",
       "  5310,\n",
       "  3313,\n",
       "  3389,\n",
       "  6224,\n",
       "  4617,\n",
       "  511,\n",
       "  8024,\n",
       "  1048,\n",
       "  4554,\n",
       "  5299,\n",
       "  1265,\n",
       "  3381,\n",
       "  5682,\n",
       "  4850,\n",
       "  8038,\n",
       "  147,\n",
       "  160,\n",
       "  145,\n",
       "  145,\n",
       "  122,\n",
       "  2477,\n",
       "  4035,\n",
       "  8020,\n",
       "  116,\n",
       "  8021,\n",
       "  510,\n",
       "  162,\n",
       "  161,\n",
       "  2208,\n",
       "  6956,\n",
       "  1146,\n",
       "  2483,\n",
       "  8020,\n",
       "  116,\n",
       "  8021,\n",
       "  510,\n",
       "  161,\n",
       "  167,\n",
       "  156,\n",
       "  8020,\n",
       "  118,\n",
       "  8021,\n",
       "  510,\n",
       "  145,\n",
       "  149,\n",
       "  143,\n",
       "  8020,\n",
       "  118,\n",
       "  8021,\n",
       "  511,\n",
       "  3318,\n",
       "  1400,\n",
       "  3389,\n",
       "  3187,\n",
       "  1265,\n",
       "  4545,\n",
       "  4881,\n",
       "  2555,\n",
       "  1400,\n",
       "  5314,\n",
       "  750,\n",
       "  124,\n",
       "  1453,\n",
       "  3309,\n",
       "  1265,\n",
       "  4545,\n",
       "  8024,\n",
       "  8024,\n",
       "  3175,\n",
       "  3428,\n",
       "  711,\n",
       "  8038,\n",
       "  1952,\n",
       "  3763,\n",
       "  1164,\n",
       "  7189,\n",
       "  122,\n",
       "  126,\n",
       "  121,\n",
       "  155,\n",
       "  149,\n",
       "  157,\n",
       "  146,\n",
       "  122,\n",
       "  8024,\n",
       "  762,\n",
       "  1383,\n",
       "  7000,\n",
       "  7159,\n",
       "  121,\n",
       "  119,\n",
       "  124,\n",
       "  149,\n",
       "  116,\n",
       "  3296,\n",
       "  1217,\n",
       "  3703],\n",
       " [1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1],\n",
       " '<PAD> O O O O O O O O B-DSE I-DSE I-DSE O O O O O O O O O O B-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-DRG I-DRG I-DRG I-DRG O O O O O O O O O B-DRG I-DRG I-DRG I-DRG O O O O O B-DRG I-DRG I-DRG',\n",
       " [0,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  8,\n",
       "  9,\n",
       "  9,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  6,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  8,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  4,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  10,\n",
       "  11,\n",
       "  11,\n",
       "  11,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  10,\n",
       "  11,\n",
       "  11,\n",
       "  11,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  10,\n",
       "  11,\n",
       "  11],\n",
       " 256)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('[CLS] ， 患 者 因 罹 患 “ 胃 癌 ” 于 2 0 1 3 - 1 0 - 2 9 在 我 院 予 行 全 麻 上 胃 癌 根 治 术 ， ， 术 中 见 ： 腹 腔 内 腹 水 ， 腹 膜 无 转 移 ， 肝 脏 未 触 及 明 显 转 移 性 灶 ， 肿 瘤 位 于 胃 体 、 胃 底 部 ， 小 弯 侧 偏 后 壁 ， 约 5 * 4 * 2 C M 大 小 ， 肿 瘤 已 侵 达 浆 膜 外 ， 第 1 、 3 组 淋 巴 结 肿 大 ， 肿 瘤 尚 能 活 动 ， 经 探 查 决 定 行 全 胃 切 除 ， 空 肠 J 字 代 胃 术 。 手 术 顺 利 ， 术 后 积 极 予 相 关 对 症 支 持 治 疗 ； ， 后 病 理 示 ： 胃 底 、 体 小 弯 侧 低 分 化 腺 癌 ， 部 分 为 印 戒 细 胞 癌 图 像 ， 蕈 伞 型 ， 面 积 5 . 2 * 3 . 5 C M ， 局 部 侵 达 粘 膜 上 层 ， 并 于 少 数 腺 管 内 查 见 癌 栓 。 双 端 切 线 及 另 送 “ 近 端 切 线 ” 未 查 见 癌 。 呈 三 组 （ 5 / 1 3 个 ） 淋 巴',\n",
       " [101,\n",
       "  8024,\n",
       "  2642,\n",
       "  5442,\n",
       "  1728,\n",
       "  5395,\n",
       "  2642,\n",
       "  100,\n",
       "  5517,\n",
       "  4617,\n",
       "  100,\n",
       "  754,\n",
       "  123,\n",
       "  121,\n",
       "  122,\n",
       "  124,\n",
       "  118,\n",
       "  122,\n",
       "  121,\n",
       "  118,\n",
       "  123,\n",
       "  130,\n",
       "  1762,\n",
       "  2769,\n",
       "  7368,\n",
       "  750,\n",
       "  6121,\n",
       "  1059,\n",
       "  7937,\n",
       "  677,\n",
       "  5517,\n",
       "  4617,\n",
       "  3418,\n",
       "  3780,\n",
       "  3318,\n",
       "  8024,\n",
       "  8024,\n",
       "  3318,\n",
       "  704,\n",
       "  6224,\n",
       "  8038,\n",
       "  5592,\n",
       "  5579,\n",
       "  1079,\n",
       "  5592,\n",
       "  3717,\n",
       "  8024,\n",
       "  5592,\n",
       "  5606,\n",
       "  3187,\n",
       "  6760,\n",
       "  4919,\n",
       "  8024,\n",
       "  5498,\n",
       "  5552,\n",
       "  3313,\n",
       "  6239,\n",
       "  1350,\n",
       "  3209,\n",
       "  3227,\n",
       "  6760,\n",
       "  4919,\n",
       "  2595,\n",
       "  4131,\n",
       "  8024,\n",
       "  5514,\n",
       "  4606,\n",
       "  855,\n",
       "  754,\n",
       "  5517,\n",
       "  860,\n",
       "  510,\n",
       "  5517,\n",
       "  2419,\n",
       "  6956,\n",
       "  8024,\n",
       "  2207,\n",
       "  2482,\n",
       "  904,\n",
       "  974,\n",
       "  1400,\n",
       "  1880,\n",
       "  8024,\n",
       "  5276,\n",
       "  126,\n",
       "  115,\n",
       "  125,\n",
       "  115,\n",
       "  123,\n",
       "  145,\n",
       "  155,\n",
       "  1920,\n",
       "  2207,\n",
       "  8024,\n",
       "  5514,\n",
       "  4606,\n",
       "  2347,\n",
       "  909,\n",
       "  6809,\n",
       "  3841,\n",
       "  5606,\n",
       "  1912,\n",
       "  8024,\n",
       "  5018,\n",
       "  122,\n",
       "  510,\n",
       "  124,\n",
       "  5299,\n",
       "  3900,\n",
       "  2349,\n",
       "  5310,\n",
       "  5514,\n",
       "  1920,\n",
       "  8024,\n",
       "  5514,\n",
       "  4606,\n",
       "  2213,\n",
       "  5543,\n",
       "  3833,\n",
       "  1220,\n",
       "  8024,\n",
       "  5307,\n",
       "  2968,\n",
       "  3389,\n",
       "  1104,\n",
       "  2137,\n",
       "  6121,\n",
       "  1059,\n",
       "  5517,\n",
       "  1147,\n",
       "  7370,\n",
       "  8024,\n",
       "  4958,\n",
       "  5499,\n",
       "  152,\n",
       "  2099,\n",
       "  807,\n",
       "  5517,\n",
       "  3318,\n",
       "  511,\n",
       "  2797,\n",
       "  3318,\n",
       "  7556,\n",
       "  1164,\n",
       "  8024,\n",
       "  3318,\n",
       "  1400,\n",
       "  4916,\n",
       "  3353,\n",
       "  750,\n",
       "  4685,\n",
       "  1068,\n",
       "  2190,\n",
       "  4568,\n",
       "  3118,\n",
       "  2898,\n",
       "  3780,\n",
       "  4545,\n",
       "  8039,\n",
       "  8024,\n",
       "  1400,\n",
       "  4567,\n",
       "  4415,\n",
       "  4850,\n",
       "  8038,\n",
       "  5517,\n",
       "  2419,\n",
       "  510,\n",
       "  860,\n",
       "  2207,\n",
       "  2482,\n",
       "  904,\n",
       "  856,\n",
       "  1146,\n",
       "  1265,\n",
       "  5593,\n",
       "  4617,\n",
       "  8024,\n",
       "  6956,\n",
       "  1146,\n",
       "  711,\n",
       "  1313,\n",
       "  2770,\n",
       "  5301,\n",
       "  5528,\n",
       "  4617,\n",
       "  1745,\n",
       "  1008,\n",
       "  8024,\n",
       "  5932,\n",
       "  835,\n",
       "  1798,\n",
       "  8024,\n",
       "  7481,\n",
       "  4916,\n",
       "  126,\n",
       "  119,\n",
       "  123,\n",
       "  115,\n",
       "  124,\n",
       "  119,\n",
       "  126,\n",
       "  145,\n",
       "  155,\n",
       "  8024,\n",
       "  2229,\n",
       "  6956,\n",
       "  909,\n",
       "  6809,\n",
       "  5111,\n",
       "  5606,\n",
       "  677,\n",
       "  2231,\n",
       "  8024,\n",
       "  2400,\n",
       "  754,\n",
       "  2208,\n",
       "  3144,\n",
       "  5593,\n",
       "  5052,\n",
       "  1079,\n",
       "  3389,\n",
       "  6224,\n",
       "  4617,\n",
       "  3410,\n",
       "  511,\n",
       "  1352,\n",
       "  4999,\n",
       "  1147,\n",
       "  5296,\n",
       "  1350,\n",
       "  1369,\n",
       "  6843,\n",
       "  100,\n",
       "  6818,\n",
       "  4999,\n",
       "  1147,\n",
       "  5296,\n",
       "  100,\n",
       "  3313,\n",
       "  3389,\n",
       "  6224,\n",
       "  4617,\n",
       "  511,\n",
       "  1439,\n",
       "  676,\n",
       "  5299,\n",
       "  8020,\n",
       "  126,\n",
       "  120,\n",
       "  122,\n",
       "  124,\n",
       "  702,\n",
       "  8021,\n",
       "  3900,\n",
       "  2349],\n",
       " [1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1],\n",
       " '<PAD> O O O O O O O B-DSE I-DSE O O O O O O O O O O O O O O O O O O O O B-OPS I-OPS I-OPS I-OPS I-OPS O O O O O O B-PAT I-PAT O B-PAT O O B-PAT O O O O O B-PAT I-PAT O O O O O O O O O O O O O O B-PAT I-PAT O B-PAT I-PAT I-PAT O B-PAT I-PAT I-PAT O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS O O O O O O O O O O O O O O O O O O O O O O O O O O B-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT I-PAT',\n",
       " [0,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  8,\n",
       "  9,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  6,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  4,\n",
       "  5,\n",
       "  1,\n",
       "  4,\n",
       "  1,\n",
       "  1,\n",
       "  4,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  4,\n",
       "  5,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  4,\n",
       "  5,\n",
       "  1,\n",
       "  4,\n",
       "  5,\n",
       "  5,\n",
       "  1,\n",
       "  4,\n",
       "  5,\n",
       "  5,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  6,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  8,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  4,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  5],\n",
       " 256)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 分段处理，使长度低于256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 256 - 2\n",
    "sents, tags_li = [], [] # list of lists\n",
    "for entry in entries:\n",
    "    words = [line.split()[0] for line in entry.splitlines()]\n",
    "    tags = ([line.split()[-1] for line in entry.splitlines()])\n",
    "#     sents.append([\"[CLS]\"] + words + [\"[SEP]\"])  # 每个句子前后加['CLS']和['SEP']\n",
    "#     tags_li.append([\"<PAD>\"] + tags + [\"<PAD>\"])\n",
    "    if len(words) > MAX_LEN:\n",
    "        # 先对句号分段\n",
    "        word, tag = [], []\n",
    "        for char, t in zip(words, tags):\n",
    "            \n",
    "            if char != '。':\n",
    "                word.append(char)\n",
    "                tag.append(t)\n",
    "            else:\n",
    "#                 if len(word) > MAX_LEN:\n",
    "#                     word = word[:MAX_LEN]\n",
    "#                     tag = tag[:MAX_LEN]\n",
    "                sents.append([\"[CLS]\"] + word[:MAX_LEN] + [\"[SEP]\"])\n",
    "                tags_li.append([\"PAD\"] + tag[:MAX_LEN] + [\"PAD\"])\n",
    "                word, tag = [], []            \n",
    "        # 最后的\n",
    "        if len(word):\n",
    "            sents.append([\"[CLS]\"] + word[:MAX_LEN] + [\"[SEP]\"])\n",
    "            tags_li.append([\"PAD\"] + tag[:MAX_LEN] + [\"PAD\"])\n",
    "            word, tag = [], []\n",
    "    else:\n",
    "        sents.append([\"[CLS]\"] + word[:MAX_LEN] + [\"[SEP]\"])\n",
    "        tags_li.append([\"PAD\"] + tag[:MAX_LEN] + [\"PAD\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6306"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6306"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tags_li)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['，',\n",
       "  '患',\n",
       "  '者',\n",
       "  '3',\n",
       "  '月',\n",
       "  '前',\n",
       "  '因',\n",
       "  '“',\n",
       "  '直',\n",
       "  '肠',\n",
       "  '癌',\n",
       "  '”',\n",
       "  '于',\n",
       "  '在',\n",
       "  '我',\n",
       "  '院',\n",
       "  '于',\n",
       "  '全',\n",
       "  '麻',\n",
       "  '上',\n",
       "  '行',\n",
       "  '直',\n",
       "  '肠',\n",
       "  '癌',\n",
       "  '根',\n",
       "  '治',\n",
       "  '术',\n",
       "  '（',\n",
       "  'D',\n",
       "  'I',\n",
       "  'X',\n",
       "  'O',\n",
       "  'N',\n",
       "  '术',\n",
       "  '）',\n",
       "  '，',\n",
       "  '手',\n",
       "  '术',\n",
       "  '过',\n",
       "  '程',\n",
       "  '顺',\n",
       "  '利',\n",
       "  '，',\n",
       "  '术',\n",
       "  '后',\n",
       "  '给',\n",
       "  '予',\n",
       "  '抗',\n",
       "  '感',\n",
       "  '染',\n",
       "  '及',\n",
       "  '营',\n",
       "  '养',\n",
       "  '支',\n",
       "  '持',\n",
       "  '治',\n",
       "  '疗',\n",
       "  '，',\n",
       "  '患',\n",
       "  '者',\n",
       "  '恢',\n",
       "  '复',\n",
       "  '好',\n",
       "  '，',\n",
       "  '切',\n",
       "  '口',\n",
       "  '愈',\n",
       "  '合',\n",
       "  '良',\n",
       "  '好',\n",
       "  '。',\n",
       "  '，',\n",
       "  '术',\n",
       "  '后',\n",
       "  '病',\n",
       "  '理',\n",
       "  '示',\n",
       "  '：',\n",
       "  '直',\n",
       "  '肠',\n",
       "  '腺',\n",
       "  '癌',\n",
       "  '（',\n",
       "  '中',\n",
       "  '低',\n",
       "  '度',\n",
       "  '分',\n",
       "  '化',\n",
       "  '）',\n",
       "  '，',\n",
       "  '浸',\n",
       "  '润',\n",
       "  '溃',\n",
       "  '疡',\n",
       "  '型',\n",
       "  '，',\n",
       "  '面',\n",
       "  '积',\n",
       "  '3',\n",
       "  '.',\n",
       "  '5',\n",
       "  '*',\n",
       "  '2',\n",
       "  'C',\n",
       "  'M',\n",
       "  '，',\n",
       "  '侵',\n",
       "  '达',\n",
       "  '外',\n",
       "  '膜',\n",
       "  '。',\n",
       "  '双',\n",
       "  '端',\n",
       "  '切',\n",
       "  '线',\n",
       "  '另',\n",
       "  '送',\n",
       "  '“',\n",
       "  '近',\n",
       "  '端',\n",
       "  '”',\n",
       "  '、',\n",
       "  '“',\n",
       "  '远',\n",
       "  '端',\n",
       "  '”',\n",
       "  '及',\n",
       "  '环',\n",
       "  '周',\n",
       "  '底',\n",
       "  '部',\n",
       "  '切',\n",
       "  '除',\n",
       "  '面',\n",
       "  '未',\n",
       "  '查',\n",
       "  '见',\n",
       "  '癌',\n",
       "  '。',\n",
       "  '肠',\n",
       "  '壁',\n",
       "  '一',\n",
       "  '站',\n",
       "  '（',\n",
       "  '1',\n",
       "  '0',\n",
       "  '个',\n",
       "  '）',\n",
       "  '、',\n",
       "  '中',\n",
       "  '间',\n",
       "  '组',\n",
       "  '（',\n",
       "  '8',\n",
       "  '个',\n",
       "  '）',\n",
       "  '淋',\n",
       "  '巴',\n",
       "  '结',\n",
       "  '未',\n",
       "  '查',\n",
       "  '见',\n",
       "  '癌',\n",
       "  '。',\n",
       "  '，',\n",
       "  '免',\n",
       "  '疫',\n",
       "  '组',\n",
       "  '化',\n",
       "  '染',\n",
       "  '色',\n",
       "  '示',\n",
       "  '：',\n",
       "  'E',\n",
       "  'R',\n",
       "  'C',\n",
       "  'C',\n",
       "  '1',\n",
       "  '弥',\n",
       "  '漫',\n",
       "  '（',\n",
       "  '+',\n",
       "  '）',\n",
       "  '、',\n",
       "  'T',\n",
       "  'S',\n",
       "  '少',\n",
       "  '部',\n",
       "  '分',\n",
       "  '弱',\n",
       "  '（',\n",
       "  '+',\n",
       "  '）',\n",
       "  '、',\n",
       "  'S',\n",
       "  'Y',\n",
       "  'N',\n",
       "  '（',\n",
       "  '-',\n",
       "  '）',\n",
       "  '、',\n",
       "  'C',\n",
       "  'G',\n",
       "  'A',\n",
       "  '（',\n",
       "  '-',\n",
       "  '）',\n",
       "  '。',\n",
       "  '术',\n",
       "  '后',\n",
       "  '查',\n",
       "  '无',\n",
       "  '化',\n",
       "  '疗',\n",
       "  '禁',\n",
       "  '忌',\n",
       "  '后',\n",
       "  '给',\n",
       "  '予',\n",
       "  '3',\n",
       "  '周',\n",
       "  '期',\n",
       "  '化',\n",
       "  '疗',\n",
       "  '，',\n",
       "  '，',\n",
       "  '方',\n",
       "  '案',\n",
       "  '为',\n",
       "  '：',\n",
       "  '奥',\n",
       "  '沙',\n",
       "  '利',\n",
       "  '铂',\n",
       "  '1',\n",
       "  '5',\n",
       "  '0',\n",
       "  'M',\n",
       "  'G',\n",
       "  'O',\n",
       "  'D',\n",
       "  '1',\n",
       "  '，',\n",
       "  '亚',\n",
       "  '叶',\n",
       "  '酸',\n",
       "  '钙',\n",
       "  '0',\n",
       "  '.',\n",
       "  '3',\n",
       "  'G',\n",
       "  '+',\n",
       "  '替',\n",
       "  '加',\n",
       "  '氟',\n",
       "  '1',\n",
       "  '.',\n",
       "  '0',\n",
       "  'G',\n",
       "  'O',\n",
       "  'D',\n",
       "  '2',\n",
       "  '-',\n",
       "  'D',\n",
       "  '6',\n",
       "  '，',\n",
       "  '同',\n",
       "  '时',\n",
       "  '给',\n",
       "  '与',\n",
       "  '升',\n",
       "  '白',\n",
       "  '细',\n",
       "  '胞',\n",
       "  '、',\n",
       "  '护',\n",
       "  '肝',\n",
       "  '、',\n",
       "  '止',\n",
       "  '吐',\n",
       "  '、',\n",
       "  '免',\n",
       "  '疫',\n",
       "  '增',\n",
       "  '强',\n",
       "  '治',\n",
       "  '疗',\n",
       "  '，',\n",
       "  '患',\n",
       "  '者',\n",
       "  '副',\n",
       "  '反',\n",
       "  '应',\n",
       "  '轻',\n",
       "  '。',\n",
       "  '院',\n",
       "  '外',\n",
       "  '期',\n",
       "  '间',\n",
       "  '患',\n",
       "  '者',\n",
       "  '一',\n",
       "  '般',\n",
       "  '情',\n",
       "  '况',\n",
       "  '好',\n",
       "  '，',\n",
       "  '无',\n",
       "  '恶',\n",
       "  '心',\n",
       "  '，',\n",
       "  '无',\n",
       "  '腹',\n",
       "  '痛',\n",
       "  '腹',\n",
       "  '胀',\n",
       "  '胀',\n",
       "  '不',\n",
       "  '适',\n",
       "  '，',\n",
       "  '无',\n",
       "  '现',\n",
       "  '患',\n",
       "  '者',\n",
       "  '为',\n",
       "  '行',\n",
       "  '复',\n",
       "  '查',\n",
       "  '及',\n",
       "  '化',\n",
       "  '疗',\n",
       "  '再',\n",
       "  '次',\n",
       "  '来',\n",
       "  '院',\n",
       "  '就',\n",
       "  '诊',\n",
       "  '，',\n",
       "  '门',\n",
       "  '诊',\n",
       "  '以',\n",
       "  '“',\n",
       "  '直',\n",
       "  '肠',\n",
       "  '癌',\n",
       "  '术',\n",
       "  '后',\n",
       "  '”',\n",
       "  '收',\n",
       "  '入',\n",
       "  '院',\n",
       "  '。',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  '近',\n",
       "  '期',\n",
       "  '患',\n",
       "  '者',\n",
       "  '精',\n",
       "  '神',\n",
       "  '可',\n",
       "  '，',\n",
       "  '饮',\n",
       "  '食',\n",
       "  '可',\n",
       "  '，',\n",
       "  '大',\n",
       "  '便',\n",
       "  '正',\n",
       "  '常',\n",
       "  '，',\n",
       "  '小',\n",
       "  '便',\n",
       "  '正',\n",
       "  '常',\n",
       "  '，',\n",
       "  '近',\n",
       "  '期',\n",
       "  '体',\n",
       "  '重',\n",
       "  '无',\n",
       "  '明',\n",
       "  '显',\n",
       "  '变',\n",
       "  '化',\n",
       "  '。'],\n",
       " ['O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'I-OPS',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'I-PAT',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-DRG',\n",
       "  'I-DRG',\n",
       "  'I-DRG',\n",
       "  'I-DRG',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-DRG',\n",
       "  'I-DRG',\n",
       "  'I-DRG',\n",
       "  'I-DRG',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-DRG',\n",
       "  'I-DRG',\n",
       "  'I-DRG',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-PAT',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-PAT',\n",
       "  'O',\n",
       "  'B-PAT',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'I-DSE',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O'])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sents[0], tags_li[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_pretrained_bert import BertModel\n",
    "bert = BertModel.from_pretrained('bert-base-chinese')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = dev_dataset[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.4081,  0.6838, -0.1342,  ..., -0.6718, -0.5864,  0.2491],\n",
       "         [-0.1821,  0.4146,  0.9377,  ..., -1.1175, -0.4919,  0.0507],\n",
       "         [-0.2332,  0.3116,  0.1658,  ...,  0.4438,  0.3029, -0.0686],\n",
       "         ...,\n",
       "         [-0.0225, -0.1823, -0.4035,  ...,  0.8034, -0.0065, -0.0556],\n",
       "         [-0.5497,  0.5112,  1.1299,  ...,  0.3418, -0.0035, -0.6358],\n",
       "         [-0.2669, -0.1594, -0.0355,  ...,  0.3744, -0.3602, -0.0103]]],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "encode_layers, _ = bert(torch.tensor(x[1]).unsqueeze(0))\n",
    "enc = encode_layers[-1]\n",
    "enc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 59, 768])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([59, 1, 768])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc = enc.view(len(x[1]), 1, -1)\n",
    "enc.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-21T10:49:15.851838Z",
     "start_time": "2019-11-21T10:49:15.810652Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils import NerDataset\n",
    "dev_dataset = NerDataset('./processed/processed_dev_bio.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-21T10:49:22.375233Z",
     "start_time": "2019-11-21T10:49:22.371889Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " '，',\n",
       " '患',\n",
       " '者',\n",
       " '2',\n",
       " '0',\n",
       " '1',\n",
       " '1',\n",
       " '年',\n",
       " '5',\n",
       " '月',\n",
       " '1',\n",
       " '日',\n",
       " '因',\n",
       " '尿',\n",
       " '频',\n",
       " '尿',\n",
       " '急',\n",
       " '在',\n",
       " '外',\n",
       " '院',\n",
       " '检',\n",
       " '查',\n",
       " '，',\n",
       " '行',\n",
       " 'B',\n",
       " '超',\n",
       " '及',\n",
       " 'C',\n",
       " 'T',\n",
       " '检',\n",
       " '查',\n",
       " '提',\n",
       " '示',\n",
       " '盆',\n",
       " '腔',\n",
       " '包',\n",
       " '块',\n",
       " '，',\n",
       " '建',\n",
       " '议',\n",
       " '进',\n",
       " '一',\n",
       " '步',\n",
       " '检',\n",
       " '查',\n",
       " '[SEP]']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_dataset.sents[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-21T10:49:26.085027Z",
     "start_time": "2019-11-21T10:49:26.081322Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<PAD>',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-INF',\n",
       " 'I-INF',\n",
       " 'O',\n",
       " 'B-INF',\n",
       " 'I-INF',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PAT',\n",
       " 'I-PAT',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " '<PAD>']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_dataset.tags_li[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-21T10:49:47.919863Z",
     "start_time": "2019-11-21T10:49:47.916869Z"
    }
   },
   "outputs": [],
   "source": [
    "for sent, tag in zip(dev_dataset.sents, dev_dataset.tags_li):\n",
    "    assert len(sent) == len(tag), f\"{len(sent)}, {len(tag)}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-21T10:42:05.114967Z",
     "start_time": "2019-11-21T10:42:05.092173Z"
    }
   },
   "outputs": [],
   "source": [
    "from pytorch_pretrained_bert import BertTokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('/root/workspace/qa_project/chinese_L-12_H-768_A-12/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-21T10:44:43.664923Z",
     "start_time": "2019-11-21T10:44:43.661980Z"
    }
   },
   "outputs": [],
   "source": [
    "VOCAB = ('<PAD>', 'O', 'B-INF', 'I-INF', 'B-PAT', 'I-PAT', 'B-OPS', \n",
    "        'I-OPS', 'B-DSE', 'I-DSE', 'B-DRG', 'I-DRG', 'B-LAB', 'I-LAB')\n",
    "tag2idx = {tag: idx for idx, tag in enumerate(VOCAB)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-21T10:49:59.867526Z",
     "start_time": "2019-11-21T10:49:59.667002Z"
    }
   },
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "366",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-4ede5fc261b1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0mis_heads\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_head\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_heads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"{idx}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m     \u001b[0midx\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: 366"
     ]
    }
   ],
   "source": [
    "idx = 0\n",
    "for words, tags in zip(dev_dataset.sents, dev_dataset.tags_li):\n",
    "    x, y = [], []\n",
    "    is_heads = []\n",
    "    for w, t in zip(words, tags):\n",
    "        tokens = tokenizer.tokenize(w) if w not in (\"[CLS]\", \"[SEP]\") else [w]\n",
    "        xx = tokenizer.convert_tokens_to_ids(tokens)\n",
    "        \n",
    "        is_head = [1] + [0]*(len(tokens) - 1)\n",
    "        t = [t] + [\"<PAD>\"] * (len(tokens) - 1)\n",
    "        yy = [tag2idx[each] for each in t]  # (T,)\n",
    "\n",
    "        x.extend(xx)\n",
    "        is_heads.extend(is_head)\n",
    "        y.extend(yy)\n",
    "    assert len(x)==len(y)==len(is_heads), f\"{idx}\"\n",
    "    idx += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-21T10:50:16.310449Z",
     "start_time": "2019-11-21T10:50:16.306665Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " '术',\n",
       " '后',\n",
       " '行',\n",
       " '2',\n",
       " '周',\n",
       " '期',\n",
       " '辅',\n",
       " '助',\n",
       " '化',\n",
       " '疗',\n",
       " '，',\n",
       " '方',\n",
       " '案',\n",
       " 'M',\n",
       " 'F',\n",
       " 'O',\n",
       " 'L',\n",
       " 'F',\n",
       " 'O',\n",
       " 'X',\n",
       " '7',\n",
       " '(',\n",
       " 'O',\n",
       " 'X',\n",
       " 'A',\n",
       " '1',\n",
       " '0',\n",
       " '0',\n",
       " 'M',\n",
       " 'G',\n",
       " '/',\n",
       " 'M',\n",
       " '\\ue236',\n",
       " ',',\n",
       " 'C',\n",
       " 'F',\n",
       " '4',\n",
       " '0',\n",
       " '0',\n",
       " 'M',\n",
       " 'G',\n",
       " '/',\n",
       " 'M',\n",
       " '\\ue236',\n",
       " ',',\n",
       " '5',\n",
       " 'F',\n",
       " 'U',\n",
       " '2',\n",
       " '4',\n",
       " '0',\n",
       " '0',\n",
       " 'M',\n",
       " 'G',\n",
       " '/',\n",
       " 'M',\n",
       " '\\ue236',\n",
       " ')',\n",
       " '，',\n",
       " '过',\n",
       " '程',\n",
       " '顺',\n",
       " '利',\n",
       " '[SEP]']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_dataset.sents[366]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CRF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x1254221b0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def argmax(vec):\n",
    "    _, idx = torch.max(vec, 1)\n",
    "    return idx.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_sequence(seq, to_ix):\n",
    "    idxs = [to_ix[w] for w in seq]\n",
    "    return torch.tensor(idxs, dtype=torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_sum_exp(vec):\n",
    "    max_score = vec[0, argmax(vec)]\n",
    "    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])\n",
    "    return max_score + \\\n",
    "        torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "START_TAG = \"<START>\"\n",
    "STOP_TAG = \"<STOP>\"\n",
    "EMBEDDING_DIM = 5\n",
    "HIDDEN_DIM = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'the': 0,\n",
       " 'wall': 1,\n",
       " 'street': 2,\n",
       " 'journal': 3,\n",
       " 'reported': 4,\n",
       " 'today': 5,\n",
       " 'that': 6,\n",
       " 'apple': 7,\n",
       " 'corporation': 8,\n",
       " 'made': 9,\n",
       " 'money': 10,\n",
       " 'georgia': 11,\n",
       " 'tech': 12,\n",
       " 'is': 13,\n",
       " 'a': 14,\n",
       " 'university': 15,\n",
       " 'in': 16}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data = [(\n",
    "    \"the wall street journal reported today that apple corporation made money\".split(),\n",
    "    \"B I I I O O O B I O O\".split()\n",
    "), (\n",
    "    \"georgia tech is a university in georgia\".split(),\n",
    "    \"B I O O O O B\".split()\n",
    ")]\n",
    "word_to_ix = {}\n",
    "for sentence, tags in training_data:\n",
    "    for word in sentence:\n",
    "        if word not in word_to_ix:\n",
    "            word_to_ix[word] = len(word_to_ix)\n",
    "word_to_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_to_ix = {\"B\": 0, \"I\": 1, \"O\": 2, START_TAG: 3, STOP_TAG: 4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([11, 12, 13, 14, 15, 16, 11])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precheck_sent = prepare_sequence(training_data[1][0], word_to_ix)\n",
    "precheck_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 1, 1, 2, 2, 2, 0, 1, 2, 2])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precheck_tags = torch.tensor([tag_to_ix[t] for t in training_data[0][1]],\n",
    "                            dtype=torch.long)\n",
    "precheck_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BiLSTM_CRF(len(word_to_ix), tag_to_ix, EMBEDDING_DIM, HIDDEN_DIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0940,  0.2521, -0.1978,  0.7061, -0.1087],\n",
       "        [ 0.0075, -0.0180, -0.3852,  0.5408, -0.0325],\n",
       "        [-0.2446,  0.3031, -0.3457,  0.4786, -0.3029],\n",
       "        [-0.2421,  0.1327, -0.3639,  0.5181, -0.2309],\n",
       "        [-0.4148,  0.1546, -0.4694,  0.4675, -0.0463],\n",
       "        [-0.2203,  0.2797, -0.2896,  0.4339, -0.6936],\n",
       "        [-0.1195,  0.1382, -0.2920,  0.4968, -0.5249]],\n",
       "       grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_feates = model._get_lstm_features(precheck_sent)\n",
    "lstm_feates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7, 5])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_feates.size()  # 5是标签长度， 7是文本长度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-10000., -10000., -10000., -10000., -10000.]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### 前向算法\n",
    "init_alphas = torch.full((1, model.tagset_size), -10000.)\n",
    "init_alphas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-10000., -10000., -10000.,      0., -10000.]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_alphas[0][model.tag_to_ix[START_TAG]] = 0.\n",
    "init_alphas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "forward_var = init_alphas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([4.3112], grad_fn=<ViewBackward>),\n",
       " tensor([5.4447], grad_fn=<ViewBackward>),\n",
       " tensor([5.2821], grad_fn=<ViewBackward>),\n",
       " tensor([-9993.8789], grad_fn=<ViewBackward>),\n",
       " tensor([4.6432], grad_fn=<ViewBackward>)]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for feat in lstm_feates:\n",
    "    alphas_t = []\n",
    "    for next_tag in range(model.tagset_size):\n",
    "        emit_score = feat[next_tag].view(1, \n",
    "                                -1).expand(1, model.tagset_size)\n",
    "        #print(emit_score)\n",
    "        #break\n",
    "        trans_score = model.transitions[next_tag].view(1, -1)\n",
    "        # print(trans_score)\n",
    "        # break\n",
    "        next_tag_var = forward_var + trans_score + emit_score\n",
    "        \n",
    "        alphas_t.append(log_sum_exp(next_tag_var).view(1))\n",
    "    # 改变\n",
    "    forward_var = torch.cat(alphas_t).view(1, -1)\n",
    "alphas_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.1195,  0.1382, -0.2920,  0.4968, -0.5249], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "terminal_var = torch.cat(alphas_t).view(1, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-1.1811e-01, -1.4420e+00, -1.1108e+00, -1.1187e+00, -1.0000e+04],\n",
       "        [-4.9566e-01, -1.9700e-01, -3.3396e-02,  1.4273e+00, -1.0000e+04],\n",
       "        [-7.5307e-01, -4.3190e-01,  6.6930e-01,  6.5051e-01, -1.0000e+04],\n",
       "        [-1.0000e+04, -1.0000e+04, -1.0000e+04, -1.0000e+04, -1.0000e+04],\n",
       "        [ 1.8568e-01, -2.7636e-01, -5.9385e-01, -3.0606e-01, -1.0000e+04]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.transitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(17, 5)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.word_embeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7, 1, 5])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeds = model.word_embeds(precheck_sent).view(len(precheck_sent), 1, -1)\n",
    "embeds.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['georgia', 'tech', 'is', 'a', 'university', 'in', 'georgia']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[-0.1786, -0.3710]],\n",
       " \n",
       "         [[-0.5775,  0.3241]]]), tensor([[[ 0.6319,  0.1522]],\n",
       " \n",
       "         [[-1.2962,  0.9093]]]))"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden = model.init_hidden()\n",
    "hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_output, hidden = model.lstm(embeds, hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7, 1, 4])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_output.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7, 4])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out = lstm_output.view(len(precheck_sent), model.hidden_dim)\n",
    "lstm_out.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7, 5])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_feates = model.hidden2tag(lstm_out)\n",
    "lstm_feates.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_pretrained_bert import BertModel\n",
    "bert = BertModel.from_pretrained('bert-base-chinese')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([11, 12, 13, 14, 15, 16, 11])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precheck_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('[CLS] ， 患 者 3 月 前 因 “ 直 肠 癌 ” 于 在 我 院 于 全 麻 上 行 直 肠 癌 根 治 术 （ D I X O N 术 ） ， 手 术 过 程 顺 利 ， 术 后 给 予 抗 感 染 及 营 养 支 持 治 疗 ， 患 者 恢 复 好 ， 切 口 愈 合 良 好 [SEP]',\n",
       " [101,\n",
       "  8024,\n",
       "  2642,\n",
       "  5442,\n",
       "  124,\n",
       "  3299,\n",
       "  1184,\n",
       "  1728,\n",
       "  100,\n",
       "  4684,\n",
       "  5499,\n",
       "  4617,\n",
       "  100,\n",
       "  754,\n",
       "  1762,\n",
       "  2769,\n",
       "  7368,\n",
       "  754,\n",
       "  1059,\n",
       "  7937,\n",
       "  677,\n",
       "  6121,\n",
       "  4684,\n",
       "  5499,\n",
       "  4617,\n",
       "  3418,\n",
       "  3780,\n",
       "  3318,\n",
       "  8020,\n",
       "  146,\n",
       "  151,\n",
       "  166,\n",
       "  157,\n",
       "  156,\n",
       "  3318,\n",
       "  8021,\n",
       "  8024,\n",
       "  2797,\n",
       "  3318,\n",
       "  6814,\n",
       "  4923,\n",
       "  7556,\n",
       "  1164,\n",
       "  8024,\n",
       "  3318,\n",
       "  1400,\n",
       "  5314,\n",
       "  750,\n",
       "  2834,\n",
       "  2697,\n",
       "  3381,\n",
       "  1350,\n",
       "  5852,\n",
       "  1075,\n",
       "  3118,\n",
       "  2898,\n",
       "  3780,\n",
       "  4545,\n",
       "  8024,\n",
       "  2642,\n",
       "  5442,\n",
       "  2612,\n",
       "  1908,\n",
       "  1962,\n",
       "  8024,\n",
       "  1147,\n",
       "  1366,\n",
       "  2689,\n",
       "  1394,\n",
       "  5679,\n",
       "  1962,\n",
       "  102],\n",
       " [1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1],\n",
       " '<PAD> O O O O O O O O B-DSE I-DSE I-DSE O O O O O O O O O O B-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O <PAD>',\n",
       " [0,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  8,\n",
       "  9,\n",
       "  9,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  6,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  7,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  0],\n",
       " 72)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utils import NerDataset\n",
    "training_data = NerDataset('./processed/processed_training_bio.txt')\n",
    "training_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['[CLS] 患 者 输 注 氨 磷 汀 恶 心 ， 停 输 [SEP]',\n",
       "  '[CLS] 自 第 下 次 化 疗 结 束 出 院 以 来 ， 患 者 未 诉 明 显 不 适 ， 一 般 情 况 保 持 良 好 ； 无 发 热 ， 无 恶 心 、 呕 吐 ， 无 反 酸 、 嗳 气 ， 无 明 显 进 食 不 适 ， 未 现 明 显 腹 痛 、 腹 胀 [SEP]',\n",
       "  '[CLS] 此 次 为 化 疗 再 次 入 院 [SEP]',\n",
       "  '[CLS] 下 次 出 院 以 来 精 神 、 睡 眠 、 饮 食 可 ， 无 腹 痛 、 腹 胀 、 发 热 ， 大 小 便 正 常 ， 体 重 较 前 无 明 显 变 化 [SEP]',\n",
       "  '[CLS] B-PAT 一 站 淋 巴 结 （ 8 个 ） 未 查 见 癌 [SEP]',\n",
       "  '[CLS] 肝 内 多 发 结 节 ， 考 虑 转 移 瘤 ， 部 分 较 前 新 发 ， 肝 S 4 病 灶 较 前 增 大 [SEP]',\n",
       "  '[CLS] 病 理 诊 断 O （ 胃 窦 ） 慢 性 萎 缩 性 胃 炎 （ 中 度 ） ， 局 部 腺 体 肠 下 皮 化 生 [SEP]',\n",
       "  '[CLS] ， 缘 于 入 院 前 4 月 余 于 我 院 诊 断 为 胃 窦 癌 ， 于 2 0 1 5 年 0 8 月 2 5 日 在 全 麻 上 行 “ 腹 腔 镜 辅 助 上 根 治 性 远 端 胃 大 部 切 除 + 残 胃 空 肠 R O U X - Y 吻 合 术 （ D 2 ） + 阑 尾 切 除 术 ” ， 手 术 顺 利 [SEP]'],\n",
       " tensor([[ 101, 2642, 5442, 6783, 3800, 3710, 4840, 3722, 2626, 2552, 8024,  977,\n",
       "          6783,  102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0],\n",
       "         [ 101, 5632, 5018,  678, 3613, 1265, 4545, 5310, 3338, 1139, 7368,  809,\n",
       "          3341, 8024, 2642, 5442, 3313, 6401, 3209, 3227,  679, 6844, 8024,  671,\n",
       "          5663, 2658, 1105,  924, 2898, 5679, 1962, 8039, 3187, 1355, 4178, 8024,\n",
       "          3187, 2626, 2552,  510, 1445, 1402, 8024, 3187, 1353, 7000,  510, 1641,\n",
       "          3698, 8024, 3187, 3209, 3227, 6822, 7608,  679, 6844, 8024, 3313, 4385,\n",
       "          3209, 3227, 5592, 4578,  510, 5592, 5515,  102,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0],\n",
       "         [ 101, 3634, 3613,  711, 1265, 4545, 1086, 3613, 1057, 7368,  102,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0],\n",
       "         [ 101,  678, 3613, 1139, 7368,  809, 3341, 5125, 4868,  510, 4717, 4697,\n",
       "           510, 7650, 7608, 1377, 8024, 3187, 5592, 4578,  510, 5592, 5515,  510,\n",
       "          1355, 4178, 8024, 1920, 2207,  912, 3633, 2382, 8024,  860, 7028, 6772,\n",
       "          1184, 3187, 3209, 3227, 1359, 1265,  102,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0],\n",
       "         [ 101,  144,  118, 9519, 8165,  671, 4991, 3900, 2349, 5310, 8020,  129,\n",
       "           702, 8021, 3313, 3389, 6224, 4617,  102,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0],\n",
       "         [ 101, 5498, 1079, 1914, 1355, 5310, 5688, 8024, 5440, 5991, 6760, 4919,\n",
       "          4606, 8024, 6956, 1146, 6772, 1184, 3173, 1355, 8024, 5498,  161,  125,\n",
       "          4567, 4131, 6772, 1184, 1872, 1920,  102,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0],\n",
       "         [ 101, 4567, 4415, 6402, 3171,  157, 8020, 5517, 4977, 8021, 2714, 2595,\n",
       "          5848, 5367, 2595, 5517, 4142, 8020,  704, 2428, 8021, 8024, 2229, 6956,\n",
       "          5593,  860, 5499,  678, 4649, 1265, 4495,  102,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "             0],\n",
       "         [ 101, 8024, 5357,  754, 1057, 7368, 1184,  125, 3299,  865,  754, 2769,\n",
       "          7368, 6402, 3171,  711, 5517, 4977, 4617, 8024,  754,  123,  121,  122,\n",
       "           126, 2399,  121,  129, 3299,  123,  126, 3189, 1762, 1059, 7937,  677,\n",
       "          6121,  100, 5592, 5579, 7262, 6774, 1221,  677, 3418, 3780, 2595, 6823,\n",
       "          4999, 5517, 1920, 6956, 1147, 7370,  116, 3655, 5517, 4958, 5499,  160,\n",
       "           157,  163,  166,  118,  167, 1431, 1394, 3318, 8020,  146,  123, 8021,\n",
       "           116, 7332, 2227, 1147, 7370, 3318,  100, 8024, 2797, 3318, 7556, 1164,\n",
       "           102]]),\n",
       " [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1]],\n",
       " ['<PAD> O O O O B-DRG I-DRG I-DRG O O O O O <PAD>',\n",
       "  '<PAD> O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-PAT O O B-PAT O <PAD>',\n",
       "  '<PAD> O O O O O O O O O <PAD>',\n",
       "  '<PAD> O O O O O O O O O O O O O O O O O B-PAT O O B-PAT O O O O O O O O O O O O O O O O O O O O <PAD>',\n",
       "  '<PAD> B-PAT I-PAT I-PAT I-PAT I-PAT I-PAT O O O O O O O O <PAD>',\n",
       "  '<PAD> B-PAT O O O O O O O O O O O O O O O O O O O B-PAT I-PAT I-PAT O O O O O O <PAD>',\n",
       "  '<PAD> O O O O O B-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE I-DSE O O O O O O O O O O <PAD>',\n",
       "  '<PAD> O O O O O O O O O O O O O O O B-DSE I-DSE I-DSE O O O O O O O O O O O O O O O O O O O B-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS O O O O O O <PAD>'],\n",
       " tensor([[ 0,  1,  1,  1,  1, 10, 11, 11,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "         [ 0,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
       "           1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
       "           1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
       "           1,  1,  1,  1,  1,  1,  1,  1,  4,  1,  1,  4,  1,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "         [ 0,  1,  1,  1,  1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "         [ 0,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
       "           4,  1,  1,  4,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
       "           1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "         [ 0,  4,  0,  0,  0,  5,  5,  5,  5,  5,  1,  1,  1,  1,  1,  1,  1,  1,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "         [ 0,  4,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
       "           1,  1,  1,  4,  5,  5,  1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "         [ 0,  1,  1,  1,  1,  1,  8,  9,  9,  9,  9,  9,  9,  9,  9,  9,  9,  9,\n",
       "           9,  9,  9,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "           0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "         [ 0,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  8,  9,\n",
       "           9,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
       "           1,  1,  6,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,\n",
       "           7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,\n",
       "           7,  7,  7,  7,  7,  7,  1,  1,  1,  1,  1,  1,  0]]),\n",
       " [14, 68, 11, 43, 19, 31, 32, 85])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for batch in train_iter:\n",
    "    data = batch\n",
    "    break\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 85])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words, x, is_heads, tags, y, seqlens = data\n",
    "x.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 85, 768])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_layers, _ = bert(x)\n",
    "enc = encoded_layers[-1]\n",
    "enc.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 85, 16])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fc = nn.Linear(768, 16)\n",
    "lstm_feats = fc(enc)\n",
    "lstm_feats.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.2317, -0.3432, -0.2300,  0.2327],\n",
       "        [ 0.0843, -0.6124,  0.0237,  0.0466],\n",
       "        [ 0.0817, -0.0319, -0.0956, -0.0518],\n",
       "        [-0.0460, -0.2713,  0.0071,  0.1377],\n",
       "        [ 0.0196, -0.1559,  0.4119,  0.0321],\n",
       "        [-0.2767,  0.0784, -0.5070,  0.1274],\n",
       "        [-0.2274, -0.2640, -0.4283,  0.2956]], grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forward_score = model._score_sentence(lstm_feates, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 1, 1, 2, 2, 2, 0, 1, 2, 2])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precheck_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-2.3651], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_score = model._score_sentence(lstm_feates, precheck_tags)\n",
    "gold_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score = torch.zeros(1)\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 0, 1, 1, 1, 2, 2, 2, 0, 1, 2, 2])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags = torch.cat([torch.tensor([model.tag_to_ix[START_TAG]],\n",
    "                              dtype=torch.long), precheck_tags])\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-1.7712], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i, feat in enumerate(lstm_feates):\n",
    "    score = score + model.transitions[tags[i+1], tags[i]] + feat[tags[i+1]]\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-2.3651], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score = score + model.transitions[model.tag_to_ix[STOP_TAG], tags[-1]]\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-25T09:03:19.347075Z",
     "start_time": "2019-11-25T09:03:17.317460Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "from crf import Bert_BiLSTM_CRF, tag2idx, log_sum_exp\n",
    "bert_crf = Bert_BiLSTM_CRF(tag2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-25T09:03:19.696728Z",
     "start_time": "2019-11-25T09:03:19.349404Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('[CLS] ， 患 者 3 月 前 因 “ 直 肠 癌 ” 于 在 我 院 于 全 麻 上 行 直 肠 癌 根 治 术 （ D I X O N 术 ） ， 手 术 过 程 顺 利 ， 术 后 给 予 抗 感 染 及 营 养 支 持 治 疗 ， 患 者 恢 复 好 ， 切 口 愈 合 良 好 [SEP]',\n",
       " [101,\n",
       "  8024,\n",
       "  2642,\n",
       "  5442,\n",
       "  124,\n",
       "  3299,\n",
       "  1184,\n",
       "  1728,\n",
       "  100,\n",
       "  4684,\n",
       "  5499,\n",
       "  4617,\n",
       "  100,\n",
       "  754,\n",
       "  1762,\n",
       "  2769,\n",
       "  7368,\n",
       "  754,\n",
       "  1059,\n",
       "  7937,\n",
       "  677,\n",
       "  6121,\n",
       "  4684,\n",
       "  5499,\n",
       "  4617,\n",
       "  3418,\n",
       "  3780,\n",
       "  3318,\n",
       "  8020,\n",
       "  146,\n",
       "  151,\n",
       "  166,\n",
       "  157,\n",
       "  156,\n",
       "  3318,\n",
       "  8021,\n",
       "  8024,\n",
       "  2797,\n",
       "  3318,\n",
       "  6814,\n",
       "  4923,\n",
       "  7556,\n",
       "  1164,\n",
       "  8024,\n",
       "  3318,\n",
       "  1400,\n",
       "  5314,\n",
       "  750,\n",
       "  2834,\n",
       "  2697,\n",
       "  3381,\n",
       "  1350,\n",
       "  5852,\n",
       "  1075,\n",
       "  3118,\n",
       "  2898,\n",
       "  3780,\n",
       "  4545,\n",
       "  8024,\n",
       "  2642,\n",
       "  5442,\n",
       "  2612,\n",
       "  1908,\n",
       "  1962,\n",
       "  8024,\n",
       "  1147,\n",
       "  1366,\n",
       "  2689,\n",
       "  1394,\n",
       "  5679,\n",
       "  1962,\n",
       "  102],\n",
       " [1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1],\n",
       " '[CLS] O O O O O O O O B-DSE I-DSE I-DSE O O O O O O O O O O B-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS I-OPS O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O [SEP]',\n",
       " [1,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  10,\n",
       "  11,\n",
       "  11,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  8,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  9,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  2],\n",
       " 72)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utils import NerDataset\n",
    "train_dataset = NerDataset('./processed/processed_training_bio.txt')\n",
    "train_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-25T09:03:19.701269Z",
     "start_time": "2019-11-25T09:03:19.698433Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from utils import pad\n",
    "train_iter = DataLoader(dataset=train_dataset,\n",
    "                       batch_size=8,\n",
    "                       shuffle=True,\n",
    "                       collate_fn=pad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-25T09:03:19.728755Z",
     "start_time": "2019-11-25T09:03:19.702858Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(['[CLS] O 现 进 食 、 体 力 可 ， 无 明 显 不 适 ， 一 般 情 况 可 [SEP]', '[CLS] 于 2 0 1 4 - 0 5 - 0 5 日 行 术 后 第 1 周 期 化 疗 ， 化 疗 方 案 为 “ X E L O X ” ， ， 具 体 为 ： 奥 沙 利 铂 （ 乐 沙 定 ） 2 4 0 M G O D 1 + 卡 陪 他 滨 片 3 片 O 2 / 日 O D 1 - 1 4 [SEP]', '[CLS] 近 1 年 来 ， 患 者 反 复 因 喘 累 、 气 促 在 我 院 住 院 治 疗 ， 予 以 改 善 循 环 、 利 尿 、 营 养 心 肌 等 对 症 治 疗 后 好 转 出 院 [SEP]', '[CLS] ， 术 中 腹 腔 冲 洗 液 送 快 速 病 理 检 查 结 果 提 示 ： （ 腹 腔 冲 洗 液 ） 离 心 涂 片 查 见 大 量 间 皮 细 胞 及 少 量 淋 巴 细 胞 、 中 性 粒 细 胞 [SEP]', '[CLS] 2 0 1 1 . 6 . 1 0 ， 复 查 C T 示 ： 肝 S 4 病 灶 ， 考 虑 转 移 瘤 [SEP]', '[CLS] 外 院 影 像 学 检 查 提 示 肝 肺 多 发 转 移 , 分 期 C T 4 N 1 M 1 I V 期 [SEP]', '[CLS] 查 无 化 疗 禁 忌 后 于 2 0 1 7 - 0 3 - 0 5 开 始 给 予 规 律 化 疗 5 周 期 ， ， 方 案 为 ： 奥 沙 利 铂 2 0 0 M G O D 1 ， 亚 叶 酸 钙 0 . 3 G + 氟 尿 嘧 啶 0 . 7 5 G O D 2 - D 6 ， 同 时 给 与 升 白 细 胞 、 护 肝 、 止 吐 、 免 疫 增 强 治 疗 ， 患 者 副 反 应 轻 [SEP]', '[CLS] 第 4 程 化 疗 后 出 现 I I I ° 血 小 板 上 降 ， 予 升 血 小 板 治 疗 后 恢 复 [SEP]'], tensor([[ 101,  157, 4385, 6822, 7608,  510,  860, 1213, 1377, 8024, 3187, 3209,\n",
      "         3227,  679, 6844, 8024,  671, 5663, 2658, 1105, 1377,  102,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0],\n",
      "        [ 101,  754,  123,  121,  122,  125,  118,  121,  126,  118,  121,  126,\n",
      "         3189, 6121, 3318, 1400, 5018,  122, 1453, 3309, 1265, 4545, 8024, 1265,\n",
      "         4545, 3175, 3428,  711,  100,  166,  147,  154,  157,  166,  100, 8024,\n",
      "         8024, 1072,  860,  711, 8038, 1952, 3763, 1164, 7189, 8020,  727, 3763,\n",
      "         2137, 8021,  123,  125,  121,  155,  149,  157,  146,  122,  116, 1305,\n",
      "         7373,  800, 4012, 4275,  124, 4275,  157,  123,  120, 3189,  157,  146,\n",
      "          122,  118,  122,  125,  102,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0],\n",
      "        [ 101, 6818,  122, 2399, 3341, 8024, 2642, 5442, 1353, 1908, 1728, 1596,\n",
      "         5168,  510, 3698,  914, 1762, 2769, 7368,  857, 7368, 3780, 4545, 8024,\n",
      "          750,  809, 3121, 1587, 2542, 4384,  510, 1164, 2228,  510, 5852, 1075,\n",
      "         2552, 5491, 5023, 2190, 4568, 3780, 4545, 1400, 1962, 6760, 1139, 7368,\n",
      "          102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0],\n",
      "        [ 101, 8024, 3318,  704, 5592, 5579, 1103, 3819, 3890, 6843, 2571, 6862,\n",
      "         4567, 4415, 3466, 3389, 5310, 3362, 2990, 4850, 8038, 8020, 5592, 5579,\n",
      "         1103, 3819, 3890, 8021, 4895, 2552, 3864, 4275, 3389, 6224, 1920, 7030,\n",
      "         7313, 4649, 5301, 5528, 1350, 2208, 7030, 3900, 2349, 5301, 5528,  510,\n",
      "          704, 2595, 5108, 5301, 5528,  102,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0],\n",
      "        [ 101,  123,  121,  122,  122,  119,  127,  119,  122,  121, 8024, 1908,\n",
      "         3389,  145,  162, 4850, 8038, 5498,  161,  125, 4567, 4131, 8024, 5440,\n",
      "         5991, 6760, 4919, 4606,  102,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0],\n",
      "        [ 101, 1912, 7368, 2512, 1008, 2110, 3466, 3389, 2990, 4850, 5498, 5511,\n",
      "         1914, 1355, 6760, 4919,  117, 1146, 3309,  145,  162,  125,  156,  122,\n",
      "          155,  122,  151,  164, 3309,  102,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0],\n",
      "        [ 101, 3389, 3187, 1265, 4545, 4881, 2555, 1400,  754,  123,  121,  122,\n",
      "          128,  118,  121,  124,  118,  121,  126, 2458, 1993, 5314,  750, 6226,\n",
      "         2526, 1265, 4545,  126, 1453, 3309, 8024, 8024, 3175, 3428,  711, 8038,\n",
      "         1952, 3763, 1164, 7189,  123,  121,  121,  155,  149,  157,  146,  122,\n",
      "         8024,  762, 1383, 7000, 7159,  121,  119,  124,  149,  116, 3703, 2228,\n",
      "         1665, 1578,  121,  119,  128,  126,  149,  157,  146,  123,  118,  146,\n",
      "          127, 8024, 1398, 3198, 5314,  680, 1285, 4635, 5301, 5528,  510, 2844,\n",
      "         5498,  510, 3632, 1402,  510, 1048, 4554, 1872, 2487, 3780, 4545, 8024,\n",
      "         2642, 5442, 1199, 1353, 2418, 6768,  102],\n",
      "        [ 101, 5018,  125, 4923, 1265, 4545, 1400, 1139, 4385,  151,  151,  151,\n",
      "          180, 6117, 2207, 3352,  677, 7360, 8024,  750, 1285, 6117, 2207, 3352,\n",
      "         3780, 4545, 1400, 2612, 1908,  102,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "            0,    0,    0,    0,    0,    0,    0]]), [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], ['[CLS] O O O O O O O O O O O O O O O O O O O O [SEP]', '[CLS] O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-DRG I-DRG I-DRG I-DRG O B-DRG I-DRG I-DRG O O O O O O O O O O B-DRG I-DRG I-DRG I-DRG I-DRG O O O O O O O O O O O O [SEP]', '[CLS] O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O [SEP]', '[CLS] O O O B-PAT I-PAT O O O O O O O O O O O O O O O O B-PAT I-PAT O O O O O O O O O O O O O O O O O O O O O O O O O O O O O [SEP]', '[CLS] O O O O O O O O O O O O B-INF I-INF O O B-PAT I-PAT I-PAT O O O O O O O O [SEP]', '[CLS] O O O O O O O O O B-DSE I-DSE I-DSE I-DSE I-DSE I-DSE O O O O O O O O O O O O O [SEP]', '[CLS] O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-DRG I-DRG I-DRG I-DRG O O O O O O O O O B-DRG I-DRG I-DRG I-DRG O O O O O B-DRG I-DRG I-DRG I-DRG O O O O O O O O O O O O O O O O O O O O O O B-PAT O O O O O O O O O O O O O O O O O [SEP]', '[CLS] O O O O O O O O O O O O B-LAB I-LAB I-LAB O O O O O B-LAB I-LAB I-LAB O O O O O [SEP]'], tensor([[ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  2,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
      "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3, 12, 13, 13, 13,  3, 12, 13, 13,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3, 12, 13, 13, 13, 13,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  2,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
      "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  2,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
      "        [ 1,  3,  3,  3,  6,  7,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  6,  7,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  2,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
      "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  4,  5,  3,  3,  6,\n",
      "          7,  7,  3,  3,  3,  3,  3,  3,  3,  3,  2,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
      "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3, 10, 11, 11, 11, 11, 11,  3,  3,\n",
      "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  2,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
      "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "         12, 13, 13, 13,  3,  3,  3,  3,  3,  3,  3,  3,  3, 12, 13, 13, 13,  3,\n",
      "          3,  3,  3,  3, 12, 13, 13, 13,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  6,  3,  3,  3,  3,  3,\n",
      "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  2],\n",
      "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, 14, 15, 15,  3,  3,\n",
      "          3,  3,  3, 14, 15, 15,  3,  3,  3,  3,  3,  2,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
      "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]]), [22, 77, 49, 54, 29, 30, 103, 30])\n"
     ]
    }
   ],
   "source": [
    "for batch in train_iter:\n",
    "    print(batch)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-25T09:03:19.743332Z",
     "start_time": "2019-11-25T09:03:19.730098Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 101,  157, 4385, 6822, 7608,  510,  860, 1213, 1377, 8024, 3187, 3209,\n",
       "         3227,  679, 6844, 8024,  671, 5663, 2658, 1105, 1377,  102,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101,  754,  123,  121,  122,  125,  118,  121,  126,  118,  121,  126,\n",
       "         3189, 6121, 3318, 1400, 5018,  122, 1453, 3309, 1265, 4545, 8024, 1265,\n",
       "         4545, 3175, 3428,  711,  100,  166,  147,  154,  157,  166,  100, 8024,\n",
       "         8024, 1072,  860,  711, 8038, 1952, 3763, 1164, 7189, 8020,  727, 3763,\n",
       "         2137, 8021,  123,  125,  121,  155,  149,  157,  146,  122,  116, 1305,\n",
       "         7373,  800, 4012, 4275,  124, 4275,  157,  123,  120, 3189,  157,  146,\n",
       "          122,  118,  122,  125,  102,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101, 6818,  122, 2399, 3341, 8024, 2642, 5442, 1353, 1908, 1728, 1596,\n",
       "         5168,  510, 3698,  914, 1762, 2769, 7368,  857, 7368, 3780, 4545, 8024,\n",
       "          750,  809, 3121, 1587, 2542, 4384,  510, 1164, 2228,  510, 5852, 1075,\n",
       "         2552, 5491, 5023, 2190, 4568, 3780, 4545, 1400, 1962, 6760, 1139, 7368,\n",
       "          102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101, 8024, 3318,  704, 5592, 5579, 1103, 3819, 3890, 6843, 2571, 6862,\n",
       "         4567, 4415, 3466, 3389, 5310, 3362, 2990, 4850, 8038, 8020, 5592, 5579,\n",
       "         1103, 3819, 3890, 8021, 4895, 2552, 3864, 4275, 3389, 6224, 1920, 7030,\n",
       "         7313, 4649, 5301, 5528, 1350, 2208, 7030, 3900, 2349, 5301, 5528,  510,\n",
       "          704, 2595, 5108, 5301, 5528,  102,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101,  123,  121,  122,  122,  119,  127,  119,  122,  121, 8024, 1908,\n",
       "         3389,  145,  162, 4850, 8038, 5498,  161,  125, 4567, 4131, 8024, 5440,\n",
       "         5991, 6760, 4919, 4606,  102,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101, 1912, 7368, 2512, 1008, 2110, 3466, 3389, 2990, 4850, 5498, 5511,\n",
       "         1914, 1355, 6760, 4919,  117, 1146, 3309,  145,  162,  125,  156,  122,\n",
       "          155,  122,  151,  164, 3309,  102,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101, 3389, 3187, 1265, 4545, 4881, 2555, 1400,  754,  123,  121,  122,\n",
       "          128,  118,  121,  124,  118,  121,  126, 2458, 1993, 5314,  750, 6226,\n",
       "         2526, 1265, 4545,  126, 1453, 3309, 8024, 8024, 3175, 3428,  711, 8038,\n",
       "         1952, 3763, 1164, 7189,  123,  121,  121,  155,  149,  157,  146,  122,\n",
       "         8024,  762, 1383, 7000, 7159,  121,  119,  124,  149,  116, 3703, 2228,\n",
       "         1665, 1578,  121,  119,  128,  126,  149,  157,  146,  123,  118,  146,\n",
       "          127, 8024, 1398, 3198, 5314,  680, 1285, 4635, 5301, 5528,  510, 2844,\n",
       "         5498,  510, 3632, 1402,  510, 1048, 4554, 1872, 2487, 3780, 4545, 8024,\n",
       "         2642, 5442, 1199, 1353, 2418, 6768,  102],\n",
       "        [ 101, 5018,  125, 4923, 1265, 4545, 1400, 1139, 4385,  151,  151,  151,\n",
       "          180, 6117, 2207, 3352,  677, 7360, 8024,  750, 1285, 6117, 2207, 3352,\n",
       "         3780, 4545, 1400, 2612, 1908,  102,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words, x, is_heads, tags, y, seqlens = batch\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-25T09:03:19.767478Z",
     "start_time": "2019-11-25T09:03:19.744685Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  2,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3, 12, 13, 13, 13,  3, 12, 13, 13,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3, 12, 13, 13, 13, 13,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  2,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  2,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "        [ 1,  3,  3,  3,  6,  7,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  6,  7,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  2,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  4,  5,  3,  3,  6,\n",
       "          7,  7,  3,  3,  3,  3,  3,  3,  3,  3,  2,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3, 10, 11, 11, 11, 11, 11,  3,  3,\n",
       "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  2,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "         12, 13, 13, 13,  3,  3,  3,  3,  3,  3,  3,  3,  3, 12, 13, 13, 13,  3,\n",
       "          3,  3,  3,  3, 12, 13, 13, 13,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  6,  3,  3,  3,  3,  3,\n",
       "          3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  2],\n",
       "        [ 1,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, 14, 15, 15,  3,  3,\n",
       "          3,  3,  3, 14, 15, 15,  3,  3,  3,  3,  3,  2,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-25T09:03:20.723780Z",
     "start_time": "2019-11-25T09:03:19.768951Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(9085.1611, grad_fn=<MeanBackward1>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score = bert_crf.neg_log_likelihood(x, y)\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-25T09:03:21.732653Z",
     "start_time": "2019-11-25T09:03:20.725992Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([184.2098, 185.2024, 185.0845, 184.9065, 185.0182, 184.8205, 186.4415,\n",
       "         184.5492], grad_fn=<MaxBackward0>),\n",
       " tensor([[ 1,  4, 11, 14,  0,  7, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,\n",
       "           7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11,\n",
       "          14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,\n",
       "           4, 11, 14,  0,  7,  4, 11, 13, 12,  4, 11, 14,  0],\n",
       "         [ 1,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11,\n",
       "          14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,\n",
       "           4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,\n",
       "           7,  4, 11, 14,  0,  7,  4,  8, 12,  4, 11, 14,  0],\n",
       "         [ 1,  4, 11, 14,  0,  7,  4, 11, 14,  0, 11, 14,  0,  7,  4, 11, 14,  0,\n",
       "           7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11,\n",
       "          14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,\n",
       "           4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0],\n",
       "         [ 1,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11,\n",
       "          14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,\n",
       "           4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 13, 12,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 13, 12,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0],\n",
       "         [ 1,  4, 11, 14,  0,  7, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 13, 12,\n",
       "           4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0],\n",
       "         [ 1,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11,\n",
       "          14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,\n",
       "           4, 11, 14,  0,  7,  4, 11, 13, 12,  4, 11, 14,  0,  7,  4, 11, 13, 12,\n",
       "           4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0],\n",
       "         [ 1,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7, 11, 14,  0,  7, 11, 14,  0,\n",
       "           7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11,\n",
       "          14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,\n",
       "           4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0],\n",
       "         [ 1,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11,\n",
       "          14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,\n",
       "           4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,\n",
       "           0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "           8, 12,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0,  7,  4,\n",
       "          11, 14,  0,  7,  4, 11, 14,  0,  7,  4, 11, 14,  0]]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_crf(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T06:46:53.715702Z",
     "start_time": "2019-11-26T06:46:53.017092Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "from crf import Bert_BiLSTM_CRF\n",
    "from utils import tag2idx, NerDataset\n",
    "from torch.utils.data import DataLoader\n",
    "train_dataset = NerDataset('./processed/processed_training_bio.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T06:46:57.986903Z",
     "start_time": "2019-11-26T06:46:53.717313Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils import pad\n",
    "train_iter = DataLoader(dataset=train_dataset,\n",
    "                       batch_size=8,\n",
    "                       shuffle=True,\n",
    "                       collate_fn=pad)\n",
    "model = Bert_BiLSTM_CRF(tag2idx).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T06:46:57.992481Z",
     "start_time": "2019-11-26T06:46:57.988792Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = 1e-4)\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T06:46:58.067308Z",
     "start_time": "2019-11-26T06:46:57.994058Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(8897.2734, device='cuda:0', grad_fn=<MeanBackward1>)\n"
     ]
    }
   ],
   "source": [
    "model.train()\n",
    "device = torch.device('cuda')\n",
    "for i, batch in enumerate(train_iter):\n",
    "    words, x, is_heads, tags, y, seqlens = batch\n",
    "    #print(x, y)\n",
    "    x = x.to(device)\n",
    "    y = y.to(device)\n",
    "    _y = y\n",
    "    loss = model.neg_log_likelihood(x, y)\n",
    "    print(loss)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T06:46:58.077751Z",
     "start_time": "2019-11-26T06:46:58.068664Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 101,  125,  119, 5541, 3491, 7755, 6574, 1872, 4495, 8024, 6956, 1146,\n",
       "         3491,  860, 3503, 2501, 1359,  100, 8024,  711, 6822,  671, 3635, 6402,\n",
       "         4545, 3119, 1057, 2769, 4906,  857, 7368,  102,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101,  123,  121,  122,  123,  118,  125,  118,  122,  129, 6121,  148,\n",
       "          157,  154,  148,  157,  166, 3175, 3428, 1265, 4545,  122, 4923, 8024,\n",
       "          678, 4923, 1265, 4545, 1400, 7755, 7767, 2829, 1169,  121, 2428, 8024,\n",
       "         4385, 2642, 5442, 6822,  671, 3635, 3780, 4545, 1057, 2769, 7368,  102],\n",
       "        [ 101, 8024, 6822,  671, 3635, 6121, 5517, 7262, 3466, 3389, 4850, 8038,\n",
       "         5517, 4977, 3971, 4550, 8024, 8024, 4567, 4415, 1726, 2845, 8038,  856,\n",
       "         1146, 1265, 5593, 4617,  102,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101,  123,  121,  122,  125,  118,  130,  118,  126, 8024, 5592, 6956,\n",
       "         5514, 4606, 3403, 2562,  131, 4617, 2834, 1333,  128,  123,  119,  125,\n",
       "          157,  128,  119,  125,  121,  163,  120,  155,  154,  102,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101, 8024, 2642, 5442, 5276,  754,  128, 2399, 1184, 1728,  100, 5310,\n",
       "         5499, 4617,  100,  754,  123,  121,  121,  130,  118,  121,  130,  118,\n",
       "          121,  129, 3189, 1762, 1059, 7937,  677, 6121, 2340, 1288, 5310, 5499,\n",
       "         1147, 7370, 3318,  102,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101,  123,  121,  122,  126,  118,  127,  157, 1355, 4385, 5498, 6760,\n",
       "         4919, 8024, 2458, 1993, 2361, 5384, 6809,  116, 3296, 5811, 1539, 5542,\n",
       "         3780, 4545,  122, 1453, 3309,  102,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101,  143,  154,  144, 8038,  125,  123,  119,  130,  149,  120,  154,\n",
       "          102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
       "        [ 101,  139,  163,  121,  121,  121,  125,  157,  157,  157, 2642, 5442,\n",
       "         6629, 4567,  809, 3341, 8024, 5125, 4868, 2213, 1377, 8024, 5517, 5287,\n",
       "         1377, 8024, 4717, 4697, 1377, 8024, 1920, 2207,  912, 3633, 2382, 8024,\n",
       "          860, 7028, 3313, 6224, 3209, 3227, 3121, 1359,  102,    0,    0,    0]],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T06:46:58.084284Z",
     "start_time": "2019-11-26T06:46:58.079117Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 48])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T06:48:04.188355Z",
     "start_time": "2019-11-26T06:48:04.098585Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 48])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_, y_hat = model(x)\n",
    "y_hat.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T06:47:32.921949Z",
     "start_time": "2019-11-26T06:47:32.918752Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 48])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T07:24:08.786843Z",
     "start_time": "2019-11-26T07:24:08.782738Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T07:26:41.084290Z",
     "start_time": "2019-11-26T07:26:40.996775Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 48, 16])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats = model._get_lstm_features(x)\n",
    "feats.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T07:26:45.240007Z",
     "start_time": "2019-11-26T07:26:45.230765Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0281, -0.0550,  0.0129,  ...,  0.0615, -0.0504,  0.0224],\n",
       "         [ 0.0049, -0.0628,  0.0059,  ...,  0.0527, -0.0570,  0.0340],\n",
       "         [ 0.0045, -0.0653, -0.0185,  ...,  0.0645, -0.0443,  0.0133],\n",
       "         ...,\n",
       "         [-0.0142,  0.0064,  0.0490,  ...,  0.0421, -0.0469,  0.0335],\n",
       "         [-0.0130, -0.0054,  0.0496,  ...,  0.0243, -0.0413,  0.0491],\n",
       "         [-0.0125, -0.0099,  0.0433,  ...,  0.0061, -0.0358,  0.0691]],\n",
       "\n",
       "        [[ 0.0536, -0.0673,  0.0237,  ...,  0.0531, -0.0357, -0.0150],\n",
       "         [ 0.0497, -0.0808,  0.0222,  ...,  0.0301, -0.0420, -0.0147],\n",
       "         [ 0.0524, -0.0685,  0.0206,  ...,  0.0186, -0.0471, -0.0395],\n",
       "         ...,\n",
       "         [ 0.0794, -0.0296,  0.0945,  ...,  0.0249, -0.0500, -0.0042],\n",
       "         [ 0.0957, -0.0198,  0.0715,  ...,  0.0150, -0.0410,  0.0012],\n",
       "         [ 0.0907, -0.0243,  0.0436,  ..., -0.0034, -0.0302,  0.0141]],\n",
       "\n",
       "        [[ 0.0062, -0.0408, -0.0195,  ...,  0.0777, -0.0781,  0.0116],\n",
       "         [-0.0052, -0.0541, -0.0196,  ...,  0.0737, -0.0927,  0.0181],\n",
       "         [ 0.0082, -0.0451, -0.0031,  ...,  0.0897, -0.0885,  0.0125],\n",
       "         ...,\n",
       "         [-0.0088,  0.0090,  0.0366,  ...,  0.0512, -0.0699,  0.0326],\n",
       "         [-0.0102,  0.0063,  0.0355,  ...,  0.0348, -0.0584,  0.0454],\n",
       "         [-0.0057,  0.0020,  0.0301,  ...,  0.0063, -0.0413,  0.0669]],\n",
       "\n",
       "        ...,\n",
       "\n",
       "        [[ 0.0028, -0.0694, -0.0178,  ...,  0.0543, -0.0547, -0.0004],\n",
       "         [-0.0079, -0.0731, -0.0236,  ...,  0.0470, -0.0586,  0.0077],\n",
       "         [ 0.0000, -0.0763, -0.0127,  ...,  0.0484, -0.0584, -0.0185],\n",
       "         ...,\n",
       "         [ 0.0114, -0.0031,  0.0283,  ...,  0.0441, -0.0594,  0.0175],\n",
       "         [ 0.0066, -0.0063,  0.0334,  ...,  0.0397, -0.0478,  0.0260],\n",
       "         [ 0.0080, -0.0198,  0.0358,  ...,  0.0196, -0.0360,  0.0482]],\n",
       "\n",
       "        [[-0.0110, -0.0268, -0.0711,  ...,  0.0624, -0.0531,  0.0613],\n",
       "         [-0.0257, -0.0256, -0.0756,  ...,  0.0495, -0.0689,  0.0841],\n",
       "         [-0.0284, -0.0421, -0.0740,  ...,  0.0480, -0.0704,  0.0754],\n",
       "         ...,\n",
       "         [-0.0625,  0.0146,  0.0058,  ...,  0.0572, -0.0654,  0.0469],\n",
       "         [-0.0590,  0.0143,  0.0146,  ...,  0.0548, -0.0626,  0.0461],\n",
       "         [-0.0386,  0.0053,  0.0130,  ...,  0.0331, -0.0471,  0.0535]],\n",
       "\n",
       "        [[ 0.0697, -0.0689, -0.0134,  ...,  0.0334, -0.0113, -0.0423],\n",
       "         [ 0.0478, -0.0712, -0.0504,  ...,  0.0363, -0.0191, -0.0334],\n",
       "         [ 0.0353, -0.0593, -0.0333,  ...,  0.0528, -0.0212, -0.0290],\n",
       "         ...,\n",
       "         [ 0.0797, -0.0038,  0.0185,  ...,  0.0141, -0.0168,  0.0108],\n",
       "         [ 0.0653,  0.0047,  0.0132,  ...,  0.0091, -0.0145,  0.0103],\n",
       "         [ 0.0511, -0.0043,  0.0066,  ..., -0.0081, -0.0136,  0.0302]]],\n",
       "       device='cuda:0', grad_fn=<ThAddBackward>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T07:27:33.161909Z",
     "start_time": "2019-11-26T07:27:33.148434Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[147.9133],\n",
       "        [147.4303],\n",
       "        [148.0974],\n",
       "        [147.7309],\n",
       "        [148.1846],\n",
       "        [147.8042],\n",
       "        [148.2599],\n",
       "        [147.7885]], device='cuda:0', grad_fn=<ThAddBackward>)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forward_score = model._forward_alg(feats)\n",
    "forward_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T08:41:59.462228Z",
     "start_time": "2019-11-26T08:41:59.456264Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]]], device='cuda:0')"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T = feats.shape[1]  # 48\n",
    "batch_size = feats.shape[0] # 8\n",
    "log_alpha = torch.Tensor(batch_size, 1, model.tagset_size).fill_(-10000.).to(device)\n",
    "log_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T08:42:01.134588Z",
     "start_time": "2019-11-26T08:42:01.131548Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 1, 16])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_alpha.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T08:42:02.111943Z",
     "start_time": "2019-11-26T08:42:02.107701Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-10000.,      0., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000.,      0., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000.,      0., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000.,      0., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000.,      0., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000.,      0., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000.,      0., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]],\n",
       "\n",
       "        [[-10000.,      0., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000., -10000., -10000., -10000., -10000., -10000.,\n",
       "          -10000., -10000.]]], device='cuda:0')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_alpha[:, 0, model.start_label_id] = 0  # start\n",
    "log_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T07:54:54.062011Z",
     "start_time": "2019-11-26T07:54:54.050858Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[  144.8641, -9855.2334,   145.5775,   145.0860,   144.8228,\n",
       "            144.7302,   145.1429,   144.9583,   145.7245,   144.4912,\n",
       "            145.4246,   145.1542,   145.1834,   145.8036,   144.8662,\n",
       "            145.2539]],\n",
       "\n",
       "        [[  144.4784, -9855.7197,   145.1395,   144.5858,   144.3716,\n",
       "            144.2122,   144.6883,   144.4500,   145.2439,   144.0007,\n",
       "            144.9102,   144.6417,   144.7353,   145.2937,   144.3879,\n",
       "            144.7162]],\n",
       "\n",
       "        [[  145.0520, -9855.0391,   145.7509,   145.2805,   145.0079,\n",
       "            144.9183,   145.3147,   145.1641,   145.9079,   144.6948,\n",
       "            145.6046,   145.3418,   145.3606,   145.9893,   145.0412,\n",
       "            145.4357]],\n",
       "\n",
       "        [[  144.7064, -9855.4033,   145.3721,   144.9241,   144.6285,\n",
       "            144.5308,   144.9641,   144.7639,   145.5438,   144.3062,\n",
       "            145.2186,   145.0020,   145.0001,   145.6365,   144.6976,\n",
       "            145.0546]],\n",
       "\n",
       "        [[  145.1684, -9854.9326,   145.8189,   145.3871,   145.0833,\n",
       "            144.9977,   145.4126,   145.2029,   146.0068,   144.7522,\n",
       "            145.6347,   145.4730,   145.4866,   146.0872,   145.1466,\n",
       "            145.5028]],\n",
       "\n",
       "        [[  144.7690, -9855.3584,   145.4650,   144.9679,   144.7246,\n",
       "            144.6329,   145.0408,   144.8705,   145.6130,   144.3780,\n",
       "            145.3037,   145.0583,   145.0720,   145.6939,   144.7512,\n",
       "            145.1250]],\n",
       "\n",
       "        [[  145.1853, -9854.8740,   145.8813,   145.4143,   145.1743,\n",
       "            145.0851,   145.5154,   145.3204,   146.0702,   144.8310,\n",
       "            145.7553,   145.4749,   145.5964,   146.1713,   145.2078,\n",
       "            145.5960]],\n",
       "\n",
       "        [[  144.7990, -9855.3447,   145.4527,   144.9843,   144.7127,\n",
       "            144.5724,   145.0228,   144.8160,   145.5982,   144.3831,\n",
       "            145.2881,   145.0463,   145.0770,   145.6561,   144.7651,\n",
       "            145.0828]]], device='cuda:0', grad_fn=<UnsqueezeBackward0>)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from crf import log_sum_exp_batch\n",
    "for t in range(1, T):\n",
    "    log_alpha = (log_sum_exp_batch(model.transitions + log_alpha, axis=-1) + \\\n",
    "                feats[:, t]).unsqueeze(1)\n",
    "log_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T07:59:33.815974Z",
     "start_time": "2019-11-26T07:59:33.812436Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[147.9133],\n",
       "        [147.4303],\n",
       "        [148.0974],\n",
       "        [147.7309],\n",
       "        [148.1846],\n",
       "        [147.8042],\n",
       "        [148.2599],\n",
       "        [147.7885]], device='cuda:0', grad_fn=<ThAddBackward>)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_prob_all_barX = log_sum_exp_batch(log_alpha)\n",
    "log_prob_all_barX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T08:20:36.376988Z",
     "start_time": "2019-11-26T08:20:36.368278Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-10000.7930, -10001.8008, -20000.0000,  ..., -10000.0947,\n",
       "          -10000.1084,  -9999.3965],\n",
       "         [-20000.0000, -20000.0000, -20000.0000,  ..., -20000.0000,\n",
       "          -20000.0000, -20000.0000],\n",
       "         [ -9997.2207, -10001.1709, -20000.0000,  ...,  -9999.9961,\n",
       "          -10000.9746,  -9999.8389],\n",
       "         ...,\n",
       "         [-10000.2920,  -9999.9307, -20000.0000,  ...,  -9999.2432,\n",
       "           -9999.7842,  -9997.3965],\n",
       "         [-10000.8906,  -9998.4971, -20000.0000,  ..., -10001.6738,\n",
       "           -9999.9238,  -9998.8242],\n",
       "         [-10000.2871,  -9998.6035, -20000.0000,  ..., -10000.3848,\n",
       "          -10000.5771, -10000.4688]],\n",
       "\n",
       "        [[-10000.7930, -10001.8008, -20000.0000,  ..., -10000.0947,\n",
       "          -10000.1084,  -9999.3965],\n",
       "         [-20000.0000, -20000.0000, -20000.0000,  ..., -20000.0000,\n",
       "          -20000.0000, -20000.0000],\n",
       "         [ -9997.2207, -10001.1709, -20000.0000,  ...,  -9999.9961,\n",
       "          -10000.9746,  -9999.8389],\n",
       "         ...,\n",
       "         [-10000.2920,  -9999.9307, -20000.0000,  ...,  -9999.2432,\n",
       "           -9999.7842,  -9997.3965],\n",
       "         [-10000.8906,  -9998.4971, -20000.0000,  ..., -10001.6738,\n",
       "           -9999.9238,  -9998.8242],\n",
       "         [-10000.2871,  -9998.6035, -20000.0000,  ..., -10000.3848,\n",
       "          -10000.5771, -10000.4688]],\n",
       "\n",
       "        [[-10000.7930, -10001.8008, -20000.0000,  ..., -10000.0947,\n",
       "          -10000.1084,  -9999.3965],\n",
       "         [-20000.0000, -20000.0000, -20000.0000,  ..., -20000.0000,\n",
       "          -20000.0000, -20000.0000],\n",
       "         [ -9997.2207, -10001.1709, -20000.0000,  ...,  -9999.9961,\n",
       "          -10000.9746,  -9999.8389],\n",
       "         ...,\n",
       "         [-10000.2920,  -9999.9307, -20000.0000,  ...,  -9999.2432,\n",
       "           -9999.7842,  -9997.3965],\n",
       "         [-10000.8906,  -9998.4971, -20000.0000,  ..., -10001.6738,\n",
       "           -9999.9238,  -9998.8242],\n",
       "         [-10000.2871,  -9998.6035, -20000.0000,  ..., -10000.3848,\n",
       "          -10000.5771, -10000.4688]],\n",
       "\n",
       "        ...,\n",
       "\n",
       "        [[-10000.7930, -10001.8008, -20000.0000,  ..., -10000.0947,\n",
       "          -10000.1084,  -9999.3965],\n",
       "         [-20000.0000, -20000.0000, -20000.0000,  ..., -20000.0000,\n",
       "          -20000.0000, -20000.0000],\n",
       "         [ -9997.2207, -10001.1709, -20000.0000,  ...,  -9999.9961,\n",
       "          -10000.9746,  -9999.8389],\n",
       "         ...,\n",
       "         [-10000.2920,  -9999.9307, -20000.0000,  ...,  -9999.2432,\n",
       "           -9999.7842,  -9997.3965],\n",
       "         [-10000.8906,  -9998.4971, -20000.0000,  ..., -10001.6738,\n",
       "           -9999.9238,  -9998.8242],\n",
       "         [-10000.2871,  -9998.6035, -20000.0000,  ..., -10000.3848,\n",
       "          -10000.5771, -10000.4688]],\n",
       "\n",
       "        [[-10000.7930, -10001.8008, -20000.0000,  ..., -10000.0947,\n",
       "          -10000.1084,  -9999.3965],\n",
       "         [-20000.0000, -20000.0000, -20000.0000,  ..., -20000.0000,\n",
       "          -20000.0000, -20000.0000],\n",
       "         [ -9997.2207, -10001.1709, -20000.0000,  ...,  -9999.9961,\n",
       "          -10000.9746,  -9999.8389],\n",
       "         ...,\n",
       "         [-10000.2920,  -9999.9307, -20000.0000,  ...,  -9999.2432,\n",
       "           -9999.7842,  -9997.3965],\n",
       "         [-10000.8906,  -9998.4971, -20000.0000,  ..., -10001.6738,\n",
       "           -9999.9238,  -9998.8242],\n",
       "         [-10000.2871,  -9998.6035, -20000.0000,  ..., -10000.3848,\n",
       "          -10000.5771, -10000.4688]],\n",
       "\n",
       "        [[-10000.7930, -10001.8008, -20000.0000,  ..., -10000.0947,\n",
       "          -10000.1084,  -9999.3965],\n",
       "         [-20000.0000, -20000.0000, -20000.0000,  ..., -20000.0000,\n",
       "          -20000.0000, -20000.0000],\n",
       "         [ -9997.2207, -10001.1709, -20000.0000,  ...,  -9999.9961,\n",
       "          -10000.9746,  -9999.8389],\n",
       "         ...,\n",
       "         [-10000.2920,  -9999.9307, -20000.0000,  ...,  -9999.2432,\n",
       "           -9999.7842,  -9997.3965],\n",
       "         [-10000.8906,  -9998.4971, -20000.0000,  ..., -10001.6738,\n",
       "           -9999.9238,  -9998.8242],\n",
       "         [-10000.2871,  -9998.6035, -20000.0000,  ..., -10000.3848,\n",
       "          -10000.5771, -10000.4688]]],\n",
       "       device='cuda:0', grad_fn=<ThAddBackward>)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_tensor = model.transitions + log_alpha\n",
    "log_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T08:22:27.445277Z",
     "start_time": "2019-11-26T08:22:27.440883Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ -9998.9668, -20000.0000,  -9997.2207,  -9998.3428,  -9998.9639,\n",
       "          -9998.6035,  -9998.2314,  -9998.5029,  -9997.5801,  -9998.6475,\n",
       "          -9997.7178,  -9997.9062,  -9998.1299,  -9997.3965,  -9998.4971,\n",
       "          -9997.1680],\n",
       "        [ -9998.9668, -20000.0000,  -9997.2207,  -9998.3428,  -9998.9639,\n",
       "          -9998.6035,  -9998.2314,  -9998.5029,  -9997.5801,  -9998.6475,\n",
       "          -9997.7178,  -9997.9062,  -9998.1299,  -9997.3965,  -9998.4971,\n",
       "          -9997.1680],\n",
       "        [ -9998.9668, -20000.0000,  -9997.2207,  -9998.3428,  -9998.9639,\n",
       "          -9998.6035,  -9998.2314,  -9998.5029,  -9997.5801,  -9998.6475,\n",
       "          -9997.7178,  -9997.9062,  -9998.1299,  -9997.3965,  -9998.4971,\n",
       "          -9997.1680],\n",
       "        [ -9998.9668, -20000.0000,  -9997.2207,  -9998.3428,  -9998.9639,\n",
       "          -9998.6035,  -9998.2314,  -9998.5029,  -9997.5801,  -9998.6475,\n",
       "          -9997.7178,  -9997.9062,  -9998.1299,  -9997.3965,  -9998.4971,\n",
       "          -9997.1680],\n",
       "        [ -9998.9668, -20000.0000,  -9997.2207,  -9998.3428,  -9998.9639,\n",
       "          -9998.6035,  -9998.2314,  -9998.5029,  -9997.5801,  -9998.6475,\n",
       "          -9997.7178,  -9997.9062,  -9998.1299,  -9997.3965,  -9998.4971,\n",
       "          -9997.1680],\n",
       "        [ -9998.9668, -20000.0000,  -9997.2207,  -9998.3428,  -9998.9639,\n",
       "          -9998.6035,  -9998.2314,  -9998.5029,  -9997.5801,  -9998.6475,\n",
       "          -9997.7178,  -9997.9062,  -9998.1299,  -9997.3965,  -9998.4971,\n",
       "          -9997.1680],\n",
       "        [ -9998.9668, -20000.0000,  -9997.2207,  -9998.3428,  -9998.9639,\n",
       "          -9998.6035,  -9998.2314,  -9998.5029,  -9997.5801,  -9998.6475,\n",
       "          -9997.7178,  -9997.9062,  -9998.1299,  -9997.3965,  -9998.4971,\n",
       "          -9997.1680],\n",
       "        [ -9998.9668, -20000.0000,  -9997.2207,  -9998.3428,  -9998.9639,\n",
       "          -9998.6035,  -9998.2314,  -9998.5029,  -9997.5801,  -9998.6475,\n",
       "          -9997.7178,  -9997.9062,  -9998.1299,  -9997.3965,  -9998.4971,\n",
       "          -9997.1680]], device='cuda:0', grad_fn=<MaxBackward0>)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.max(log_tensor, -1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T08:43:02.420484Z",
     "start_time": "2019-11-26T08:43:02.417108Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 1, 16])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_alpha.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T11:35:56.230572Z",
     "start_time": "2019-11-26T11:35:56.129678Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 48, 768])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc = model._bert_enc(x)\n",
    "enc.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T11:36:19.699545Z",
     "start_time": "2019-11-26T11:36:19.696053Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 46, 768])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc[:, 1:-1, :].size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T11:38:16.228361Z",
     "start_time": "2019-11-26T11:38:16.206470Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 46, 16])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc, _ = model.lstm(enc[:, 1:-1, :])\n",
    "feats = model.fc(enc)\n",
    "feats.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T11:38:47.509457Z",
     "start_time": "2019-11-26T11:38:47.498534Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 99.8801, 100.5272,  99.8877,  99.9009,  99.8440, 100.6436, 100.8326,\n",
       "        101.1094], device='cuda:0', grad_fn=<MaxBackward0>)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score, tag_seq = model._viterbi_decode(feats)\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-26T11:38:53.741395Z",
     "start_time": "2019-11-26T11:38:53.737457Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10],\n",
       "        [ 1,  6, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "         10, 10, 10, 10, 10, 10, 10, 10, 10, 10]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
